package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"time"
	//"io"
	"io/ioutil"
	"math/rand"
	"regexp"
	"strconv"
	"strings"
)

const max_routine_num int = 1
const max_list_node_num int = 100000

var (
	h         bool
	g_cur_dir                    string                                                           //当前目录
	g_dst_pic_dir                string                                                           //保存图片的目录
	start                        string                                                           //开始的url
	gr                           *rand.Rand     = rand.New(rand.NewSource(time.Now().UnixNano())) //全局随机数据
	g_routine_num                int            = 0                                               //协和个数
	ch_msg_notify                chan string    = make(chan string)                               //通知主协程子协和是否退出和生成，方便管理个数
	ch_msg_url                   chan string    = make(chan string)                               //通知主协程有新的url请求加入
	ch_msg_url_pic               chan string    = make(chan string)                               //通知主协程有新的图片url
	ch_msg_url_repeat            chan string    = make(chan string)                               //通知主协程有url需要重新加入
	g_url_list                   *List          = &List{0, nil, nil}                              //url链表
	g_url_pic_list               *List          = &List{0, nil, nil}                              //url图片链表，需要优先处理
	g_repeat_map                 map[string]int = make(map[string]int, 100000)                    //用来做重复判断的map
	g_too_many_request           string         = "Too Many Request"                              //可能是发太快，需要限速延时
	g_too_many_request_sleep     int            = 100                                             //100ms，url请求前的睡眠时间
	g_too_many_request_sleep_max int            = 1000                                            //最大500ms
)

var headers = [...]string{
	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
	"Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11",
	"Opera/9.25 (Windows NT 5.1; U; en)",
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
	"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12",
	"Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9",
	"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
	"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0",
	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36"}

func init() {
	flag.BoolVar(&h, "h", false, "帮助信息")
	flag.BoolVar(&h, "H", false, "帮助信息")

	flag.StringVar(&start, "s", "www.mzitu.com", "start url")

	flag.Usage = usage

	flag.Parse()

	fmt.Println("first url=", start)
}

func main() {
	flag.Parse()
	if h {
		flag.Usage()
		return
	}

	g_cur_dir, _ = os.Getwd()
	g_dst_pic_dir = g_cur_dir + "/pic"
	os.RemoveAll(g_dst_pic_dir)
	os.MkdirAll(g_dst_pic_dir, 0777)
	os.MkdirAll(g_dst_pic_dir, 0777)
	//begin
	go parseUrl("http://" + start)

	for {
		select {
		case v := <-ch_msg_notify:
			if v == "enter" {
				g_routine_num++
				//fmt.Println("add routine g_routine_num", g_routine_num)
			} else if v == "exit" {
				g_routine_num--
				//fmt.Println("del routine g_routine_num", g_routine_num)
			} else {
				//fmt.Println("recv err ch_msg_notify ", v)
				g_routine_num = 0
			}
		case v := <-ch_msg_url:
			//fmt.Println("recv new url ", v)
			if !is_valid_url(v) {
				//fmt.Println("invalid url", v)
				break
			}
			//fmt.Println("valid url", v)
			list_add(g_url_list, v)
			//fmt.Println("url list add ", g_url_list.length)
		case v := <-ch_msg_url_pic:
			list_add(g_url_pic_list, v)
		case v := <-ch_msg_url_repeat:
			fmt.Println("url need repeat ", v, "g_too_many_request_sleep", g_too_many_request_sleep, "g_too_many_request_sleep_max", g_too_many_request_sleep_max)
			if g_too_many_request_sleep < g_too_many_request_sleep_max {
				g_too_many_request_sleep += 100
				fmt.Println("g_too_many_request_sleep=", g_too_many_request_sleep)
			}
			g_repeat_map[v] = 0
			if isPic(v) {
				list_add(g_url_pic_list, v)
			} else {
				list_add(g_url_list, v)
			}
		}

		//看需要开户新的协程处理
		new_routine_check()

		//fmt.Println("g_url_pic_list", g_url_pic_list.length, "g_url_list", g_url_list.length, "g_routine_num", g_routine_num)
		if g_url_pic_list.length == 0 &&
			g_url_list.length == 0 &&
			g_routine_num == 0 {
			fmt.Println("proc over now")
			break
		}
	}
}

func list_add(list *List, str string) {
	if list.length >= max_list_node_num {
		return
	}

	list.Append(&Node{str, nil})
}

func is_valid_url(url string) bool {
	if strings.Contains(url, "meizitu") ||
		strings.Contains(url, "mzitu") {
		return true
	}

	return false
}

func new_routine_check() {
	for {
		if g_routine_num >= max_routine_num {
			break
		}
		//优先pic url
		node := g_url_pic_list.GetNext()
		if node == nil {
			node = g_url_list.GetNext()
		}
		if node == nil {
			break
		}
		//判断下当前url是否已经处理过，用map处理
		if g_repeat_map[node.value] > 0 {
			continue
		}
		g_repeat_map[node.value] = 1
		//fmt.Println("go a new routine", g_routine_num, "urls", g_url_list.length, "dst url", node.value)
		go parseUrl(node.value)
		break
	}
}

func parseUrl(url string) {
	//进入
	ch_msg_notify <- "enter"

	//退出
	defer func() {
		ch_msg_notify <- "exit"
	}()

	//睡眠一下，不然服务器可能回too many request ......
	time.Sleep(time.Duration(g_too_many_request_sleep) * time.Millisecond)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println("NewRequest failed")
		return
	}
	//说是可以处理短连接，大量复连接时可能被服务器拒绝的问题
	// req.Close = true
	// req.Header.Add("Connection", "close")
	//di := gr.Int()%len(headers)
	//req.Header.Add("User-Agent", headers[di])
	req.Header.Add("User-Agent", strconv.Itoa(gr.Int()))
	//防盗图处理
	//req.Header.Set("Referer", "https://i5.meizitu.net")
	req.Header.Set("Referer", "baidu")
	client := &http.Client{Timeout: 10 * time.Second}

	fmt.Println("pl", g_url_pic_list.length, "l", g_url_list.length, "go for url", url)
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println("client do failed", err)
		return
	}
	defer resp.Body.Close()

	byte, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println("ioutil readall err", err)
		return
	}
	//这里判断下有没有出现too many request问题
	if strings.Contains(string(byte), g_too_many_request) {
		ch_msg_url_repeat <- url
		return
	}
	//这里判断一下，如果是图片url，则直接保存图片
	if isPic(url) {
		fmt.Println("proc pic ", url, "len", len(byte))
		rs := strings.Split(url, "/")
		filename := g_dst_pic_dir + "/" + rs[len(rs)-1]
		imagefile, err := os.Create(filename)
		if err != nil {
			fmt.Println("os create failed ", filename, "err", err)
			return
		}
		imagefile.Write(byte)
		imagefile.Close()
		return
	}

	parseHtml(url, byte)
}

func isPic(url string) bool {
	if strings.HasSuffix(url, ".jpg") ||
		strings.HasSuffix(url, ".png") {
		return true
	}

	return false
}

func parseHtml(url string, bytes []byte) {
	//fmt.Println(string(bytes))
	//将所有的url都抓出来
	//src="https://www.mzitu.com/static/pc/push.js?1202"></script>
	reg := regexp.MustCompile("https://[^\"']+")
	dataSlice := reg.FindAll(bytes, -1)
	var num int = 0
	for _, v := range dataSlice {
		num++
		if isPic(string(v)) {
			ch_msg_url_pic <- string(v)
		} else {
			ch_msg_url <- string(v)
		}
	}
	// fmt.Println("url", url, "get urlnums", num)
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage:Clib[-hH] [-s url]\n")

	flag.PrintDefaults()
}

//实现个简单单链表
type Node struct {
	value string
	next  *Node
}

type List struct {
	length int
	start  *Node
	end    *Node
}

func (list *List) Append(node *Node) {
	if list.length == 0 {
		list.start = node
	} else {
		list.end.next = node
	}

	list.end = node
	node.next = nil
	list.length++
}

func (list *List) GetNext() *Node {
	if list.length == 0 {
		return nil
	}
	t := list.start
	list.start = t.next
	list.length--

	return t
}
